import os
import os.path
import re
import time
import requests
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor, wait

links = []
headers = {
    #"referer": "https://bing.lylares.com/",
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36 Edg/85.0.564.51"
}

#获取特征码
def get_code(page):
    api_url = "https://bing.lylares.com/web/api"
    form = {
        "append": "list-home",
        "paged": page,
        "action": "ajax_load_posts",
        "query": "", 
        "page": "home"
    }

    response = requests.post(api_url, headers=headers, data=form)
    soup = BeautifulSoup(response.content, "lxml")
    code_list = [i.get("href") for i in soup.select("a[class='media-content']")]
    for i in code_list:
        links.append(i)


def down(url):
    #构造文件名
    index = "https://bing.lylares.com/"
    response = requests.get(index+url, headers=headers)
    soup = BeautifulSoup(response.content, "lxml")
    title = re.split("[|©]", soup.title.get_text())[0]
    date = soup.select("time[class^='date']")[0].get_text().strip()
    name = f"{date}-{title}.jpg"
    
    #保存图片
    down_index = "https://bing.lylares.com/download/hd?id="
    pic_url = down_index + re.split("[/.]", url)[1]
    pic_response = requests.get(pic_url, headers=headers, stream=True)
    with open(name, "wb") as f:
        for i in pic_response.iter_content(128):
            f.write(i)
    print(f"{date} Download Success")


def enter():
    os.chdir(r"D:\code\python\crawl\bing_lylares_com")
    floder_name = "pics"
    if not os.path.exists(floder_name):
        os.mkdir(floder_name)
    os.chdir(floder_name)


def main():
    start = time.time()
    enter()
    
    executor = ThreadPoolExecutor(max_workers=5)
    tasks = [executor.submit(get_code,(i)) for i in range(1,6)]
    wait(tasks)
    
    tasks2 = [executor.submit(down,(i)) for i in links]
    wait(tasks2)
    
    print(f"Totol Spend {time.time()-start}")

if __name__ == "__main__":
    main()